# @Time : 2021/10/25 15:16
# @Author : Li Kunlun
# @Description : 生产者和消费者模块
import chapter08.spider02.blog_spider as blog_spider
import threading
import time
import random
import queue


def do_craw(url_queue: queue.Queue, html_queue: queue.Queue):
    while True:
        url = url_queue.get()
        html = blog_spider.craw(url)
        # 将下载的html结果放到html_queue队列中
        html_queue.put(html)
        print(threading.current_thread().name, f"craw {url}", f"url_queue.size=", url_queue.qsize())
        # 随机休息1-2秒
        time.sleep(random.randint(1, 2))


def do_parse(html_queue: queue.Queue, fout):
    while True:
        html = html_queue.get()
        # 获取对象之后进行解析
        results = blog_spider.parse(html)
        for result in results:
            # 文件对象fout
            fout.write(str(result) + "\n")
        print(threading.current_thread().name, f"results.size=", len(results), "html_queue.size=", html_queue.qsize())
        # 随机休息1-2秒
        time.sleep(random.randint(1, 2))


if __name__ == '__main__':
    url_queue = queue.Queue()
    html_queue = queue.Queue()
    for url in blog_spider.urls:
        url_queue.put(url)

    for idx in range(3):
        t = threading.Thread(target=do_craw, args=(url_queue, html_queue), name=f"craw{idx}")
        t.start()

    fout = open("data.txt", "w")

    for idx in range(2):
        t = threading.Thread(target=do_parse, args=(html_queue, fout), name=f"parse{idx}")
        t.start()
